{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment: \n",
    "\n",
    "Evaluate hebbian growth\n",
    "\n",
    "#### Motivation:\n",
    "\n",
    "Control all other variables and evaluate hebbian growth alone\n",
    "\n",
    "#### Conclusions:\n",
    "\n",
    "- Lower accuracy, about 0.2%, but it converges faster (18 vs 24) compared to random growth.\n",
    "- Results are consistent with expectation: it accelerates learning, but focusing early on few specific units might prevent the neural network from finding a different set of connections that can lead to higher performance\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from dynamic_sparse.common.browser import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and check data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = ['neurips_debug_test13', ]\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>momentum</th>\n",
       "      <th>network</th>\n",
       "      <th>num_classes</th>\n",
       "      <th>on_perc</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th>test_noise</th>\n",
       "      <th>use_kwinners</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_prune_perc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_hebbian_grow=True</td>\n",
       "      <td>0.989650</td>\n",
       "      <td>27</td>\n",
       "      <td>0.926333</td>\n",
       "      <td>0</td>\n",
       "      <td>0.987450</td>\n",
       "      <td>0.989083</td>\n",
       "      <td>0.9783</td>\n",
       "      <td>16</td>\n",
       "      <td>0.9618</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_hebbian_grow=False</td>\n",
       "      <td>0.992483</td>\n",
       "      <td>27</td>\n",
       "      <td>0.923767</td>\n",
       "      <td>0</td>\n",
       "      <td>0.989317</td>\n",
       "      <td>0.991917</td>\n",
       "      <td>0.9810</td>\n",
       "      <td>19</td>\n",
       "      <td>0.9622</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_hebbian_grow=True</td>\n",
       "      <td>0.990167</td>\n",
       "      <td>28</td>\n",
       "      <td>0.925483</td>\n",
       "      <td>0</td>\n",
       "      <td>0.986908</td>\n",
       "      <td>0.989100</td>\n",
       "      <td>0.9795</td>\n",
       "      <td>22</td>\n",
       "      <td>0.9648</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_hebbian_grow=False</td>\n",
       "      <td>0.992417</td>\n",
       "      <td>25</td>\n",
       "      <td>0.926733</td>\n",
       "      <td>0</td>\n",
       "      <td>0.989283</td>\n",
       "      <td>0.992067</td>\n",
       "      <td>0.9813</td>\n",
       "      <td>18</td>\n",
       "      <td>0.9605</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_hebbian_grow=True</td>\n",
       "      <td>0.989933</td>\n",
       "      <td>23</td>\n",
       "      <td>0.927350</td>\n",
       "      <td>0</td>\n",
       "      <td>0.986875</td>\n",
       "      <td>0.989267</td>\n",
       "      <td>0.9780</td>\n",
       "      <td>7</td>\n",
       "      <td>0.9625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Experiment Name  train_acc_max  train_acc_max_epoch  train_acc_min  \\\n",
       "0   0_hebbian_grow=True       0.989650                   27       0.926333   \n",
       "1  1_hebbian_grow=False       0.992483                   27       0.923767   \n",
       "2   2_hebbian_grow=True       0.990167                   28       0.925483   \n",
       "3  3_hebbian_grow=False       0.992417                   25       0.926733   \n",
       "4   4_hebbian_grow=True       0.989933                   23       0.927350   \n",
       "\n",
       "   train_acc_min_epoch  train_acc_median  train_acc_last  val_acc_max  \\\n",
       "0                    0          0.987450        0.989083       0.9783   \n",
       "1                    0          0.989317        0.991917       0.9810   \n",
       "2                    0          0.986908        0.989100       0.9795   \n",
       "3                    0          0.989283        0.992067       0.9813   \n",
       "4                    0          0.986875        0.989267       0.9780   \n",
       "\n",
       "   val_acc_max_epoch  val_acc_min  ...  momentum  network  num_classes  \\\n",
       "0                 16       0.9618  ...       0.9   MLPHeb           10   \n",
       "1                 19       0.9622  ...       0.9   MLPHeb           10   \n",
       "2                 22       0.9648  ...       0.9   MLPHeb           10   \n",
       "3                 18       0.9605  ...       0.9   MLPHeb           10   \n",
       "4                  7       0.9625  ...       0.9   MLPHeb           10   \n",
       "\n",
       "   on_perc optim_alg  pruning_early_stop  test_noise  use_kwinners  \\\n",
       "0      0.2       SGD                   0       False         False   \n",
       "1      0.2       SGD                   0       False         False   \n",
       "2      0.2       SGD                   0       False         False   \n",
       "3      0.2       SGD                   0       False         False   \n",
       "4      0.2       SGD                   0       False         False   \n",
       "\n",
       "  weight_decay weight_prune_perc  \n",
       "0       0.0001               0.3  \n",
       "1       0.0001               0.3  \n",
       "2       0.0001               0.3  \n",
       "3       0.0001               0.3  \n",
       "4       0.0001               0.3  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace hebbian prine\n",
    "df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)\n",
    "df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'epochs',\n",
       "       'experiment_file_name', 'trial_time', 'mean_epoch_time', 'batch_norm',\n",
       "       'data_dir', 'dataset_name', 'debug_sparse', 'debug_weights', 'device',\n",
       "       'hebbian_grow', 'hebbian_prune_perc', 'hidden_sizes', 'input_size',\n",
       "       'learning_rate', 'lr_gamma', 'lr_milestones', 'lr_scheduler', 'model',\n",
       "       'momentum', 'network', 'num_classes', 'on_perc', 'optim_alg',\n",
       "       'pruning_early_stop', 'test_noise', 'use_kwinners', 'weight_decay',\n",
       "       'weight_prune_perc'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16, 42)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name                                      1_hebbian_grow=False\n",
       "train_acc_max                                                    0.992483\n",
       "train_acc_max_epoch                                                    27\n",
       "train_acc_min                                                    0.923767\n",
       "train_acc_min_epoch                                                     0\n",
       "train_acc_median                                                 0.989317\n",
       "train_acc_last                                                   0.991917\n",
       "val_acc_max                                                         0.981\n",
       "val_acc_max_epoch                                                      19\n",
       "val_acc_min                                                        0.9622\n",
       "val_acc_min_epoch                                                       0\n",
       "val_acc_median                                                    0.97855\n",
       "val_acc_last                                                       0.9797\n",
       "epochs                                                                 30\n",
       "experiment_file_name    /Users/lsouza/nta/results/neurips_debug_test13...\n",
       "trial_time                                                        12.2084\n",
       "mean_epoch_time                                                  0.406945\n",
       "batch_norm                                                           True\n",
       "data_dir                                        /home/ubuntu/nta/datasets\n",
       "dataset_name                                                        MNIST\n",
       "debug_sparse                                                         True\n",
       "debug_weights                                                        True\n",
       "device                                                               cuda\n",
       "hebbian_grow                                                        False\n",
       "hebbian_prune_perc                                                      0\n",
       "hidden_sizes                                                          100\n",
       "input_size                                                            784\n",
       "learning_rate                                                         0.1\n",
       "lr_gamma                                                              0.1\n",
       "lr_milestones                                                          60\n",
       "lr_scheduler                                                  MultiStepLR\n",
       "model                                                        DSNNMixedHeb\n",
       "momentum                                                              0.9\n",
       "network                                                            MLPHeb\n",
       "num_classes                                                            10\n",
       "on_perc                                                               0.2\n",
       "optim_alg                                                             SGD\n",
       "pruning_early_stop                                                      0\n",
       "test_noise                                                          False\n",
       "use_kwinners                                                        False\n",
       "weight_decay                                                       0.0001\n",
       "weight_prune_perc                                                     0.3\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "DSNNMixedHeb    16\n",
       "Name: model, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('model')['model'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment Details"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "base_exp_config = dict(\n",
    "    device=\"cuda\",\n",
    "    # dataset related\n",
    "    dataset_name=\"MNIST\",\n",
    "    data_dir=os.path.expanduser(\"~/nta/datasets\"),\n",
    "    input_size=784,\n",
    "    num_classes=10,\n",
    "    # network related\n",
    "    network=\"MLPHeb\",\n",
    "    hidden_sizes=[100, 100, 100],\n",
    "    batch_norm=True,\n",
    "    use_kwinners=False,\n",
    "    # model related\n",
    "    model=\"DSNNMixedHeb\",\n",
    "    on_perc=0.2,\n",
    "    optim_alg=\"SGD\",\n",
    "    momentum=0.9,\n",
    "    weight_decay=1e-4,    \n",
    "    learning_rate=0.1,\n",
    "    lr_scheduler=\"MultiStepLR\",\n",
    "    lr_milestones=[30,60,90],\n",
    "    lr_gamma=0.1,\n",
    "    # sparse related\n",
    "    hebbian_prune_perc=None,\n",
    "    weight_prune_perc=0.3,\n",
    "    pruning_early_stop=0,\n",
    "    hebbian_grow=tune.grid_search([True, False]),\n",
    "    # additional validation\n",
    "    test_noise=False,\n",
    "    # debugging\n",
    "    debug_weights=True,\n",
    "    debug_sparse=True,\n",
    ")\n",
    "\n",
    "# ray configurations\n",
    "tune_config = dict(\n",
    "    name=__file__.replace(\".py\", \"\") + \"_test13\",\n",
    "    num_samples=8,\n",
    "    local_dir=os.path.expanduser(\"~/nta/results\"),\n",
    "    checkpoint_freq=0,\n",
    "    checkpoint_at_end=False,\n",
    "    stop={\"training_iteration\": 30},\n",
    "    resources_per_trial={\"cpu\": 1, \"gpu\": 0.25},\n",
    "    loggers=DEFAULT_LOGGERS,\n",
    "    verbose=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Did any  trials failed?\n",
    "df[df[\"epochs\"]<30][\"epochs\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16, 42)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing failed or incomplete trials\n",
    "df_origin = df.copy()\n",
    "df = df_origin[df_origin[\"epochs\"]>=30]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: epochs, dtype: int64)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# which ones failed?\n",
    "# failed, or still ongoing?\n",
    "df_origin['failed'] = df_origin[\"epochs\"]<30\n",
    "df_origin[df_origin['failed']]['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.3f} ± {:.3f}\".format(s.mean(), s.std())\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,\n",
    "                   'val_acc_last': stats,\n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'val_acc_last': stats,\n",
    "                   'model': ['count']})).round(round)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### What are optimal levels of hebbian and weight pruning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_last</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_grow</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>24</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.001</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>18</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.002</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             val_acc_max_epoch val_acc_max                      val_acc_last  \\\n",
       "                    round_mean         min    max   mean    std          min   \n",
       "hebbian_grow                                                                   \n",
       "False                       24       0.979  0.981  0.981  0.001        0.977   \n",
       "True                        18       0.978  0.980  0.979  0.001        0.974   \n",
       "\n",
       "                                 model  \n",
       "               max   mean    std count  \n",
       "hebbian_grow                            \n",
       "False         0.98  0.979  0.001     8  \n",
       "True          0.98  0.976  0.002     8  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_grow'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Lower accuracy, about 0.2%, but it converges faster (18 vs 24) compared to random growth.\n",
    "- Results are consistent with expectation: it accelerates learning, but focusing early on few specific units might prevent the neural network from finding a different set of connections that can lead to higher performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
